"""
Case Type   : 极致RTO-硬件故障类
Case Name   : 开启极致RTO，业务压力后备机进行switchover过程中备机网卡停止3分钟
Create At   : 2020/1/5
Owner       : 5328113
Description :
    1、开启极致RTO
    2、备机进行switchover
    3、备机网卡停止3分钟
    4、switchover结果
    5、switchover失败后，重启主备；
    6、再次进行switchover
    7、还原主备，原主机进行switchover
    8、执行完成后，关闭极致RTO
Expect      :
    1、开启极致RTO成功
    2、备机开始进行switchover
    3、备机网卡停止3分钟
    4、switchover结果
    5、switchover失败后，重启主备；
    6、再次进行switchover
    7、还原主备，原主机进行switchover
    8、执行完成后，关闭极致RTO
History     :
    modified：2021-9-3 by 5328113;cfe注入故障返回结果变更，优化断言；
    modified：2021-4-22 by 5328113;状态结果查询方式变更，修改脚本；
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified 2021-03-01 by 5328113；修改获取网卡逻辑
    modified 2024-07-10 by peilinqian；注入网卡故障，switchover可能成功，也可能失败，优化用例逻辑
"""

import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(f"-----{os.path.basename(__file__)} start-----")
        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、备机进行switchover----')
            switch_thread = ComThread(self.sta_1_sh.executeGsctl,
                                      args=('switchover',
                                            self.constant.SWITCH_SUCCESS_MSG,
                                            '',
                                            False))
            switch_thread.setDaemon(True)
            switch_thread.start()

            self.log.info('----3、停止DN备机网卡3分钟----')
            self.log.info('----获取网卡----')
            network_card_cmd = 'ifconfig'
            self.log.info(network_card_cmd)
            network_card_result = self.sta1_root.sh(network_card_cmd).result()
            network_card_list = network_card_result.split('\n\n')
            self.log.info(network_card_list)
            network_card = ''
            for network_card_item in network_card_list:
                if self.sta1_dbuser.db_host in network_card_item:
                    network_card = network_card_item.split(':')[0].strip()
                    self.log.info(network_card)

            self.log.info('----注入故障----')
            inject_cmd = f'rNet_down (dev, time)values({network_card}, 180)'
            self.log.info(inject_cmd)
            inject_result = self.com.cfe_inject(self.sta1_root, inject_cmd)
            self.log.info(inject_result)
            self.assertIn(self.constant.cfe_inject_netdown_success_msg,
                          inject_result)
            start_time = time.time()

            self.log.info('----4、switchover结果----')
            switch_thread.join()
            switch_result = switch_thread.get_result()
            self.log.info(switch_result)
            if not switch_result:
                self.log.info('----查询主备状态----')
                self.pri_sh.getDbClusterStatus('status')
                time.sleep(5)

                self.log.info('----5、switchover失败后，重启主备；----')
                self.pri_sh.stopDbCluster()
                time.sleep(2)
                self.pri_sh.startDbCluster()

                self.log.info('----查询主备状态----')
                status = self.pri_sh.getDbClusterStatus('status')
                self.assertTrue(status)

                self.log.info('----6、再次进行switchover----')
                success_msg = self.constant.SWITCH_SUCCESS_MSG
                switch_result = self.sta_1_sh.executeGsctl('switchover',
                                                           success_msg)
                self.assertTrue(switch_result)

                self.log.info('----查询主备状态----')
                self.pri_sh.getDbClusterStatus('status')
                time.sleep(5)

                self.log.info('----进行refresh----')
                refresh_msg = self.sta_1_sh.exec_refresh_conf()
                self.assertTrue(refresh_msg)
            else:
                end_time = time.time()
                self.log.info('start_time:' + str(start_time) +
                              ';end_time:' + str(end_time))
                self.assertGreaterEqual(end_time - start_time, 180,
                                        "switchover结果成功，执行时间大于故障时间")
                self.log.info('----跳过第5、6步骤----')

            self.log.info('----查看主机query，同步是否正常----')
            self.sta_1_sh.check_location_consistency('primary',
                                                     self.node_num,
                                                     self.min_time)
            self.log.info('----查询主备状态----')
            status = self.sta_1_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----7、还原主备，原主机进行switchover----')
            success_msg = self.constant.SWITCH_SUCCESS_MSG
            switch_result = self.pri_sh.executeGsctl('switchover',
                                                     success_msg)
            self.assertTrue(switch_result)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            self.log.info('----进行refresh----')
            refresh_msg = self.pri_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----8、执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

    def tearDown(self):
        self.log.info('----this is tearDown----')

        host_name = self.pri_root.sh('hostname').result()
        if self.pri_sh.check_whether_need_switch(host_name):
            self.log.info('----恢复主备关系----')
            if self.pri_sh.check_whether_need_build():
                self.log.info('如果备机无法恢复，需要重建备机')
                self.pri_sh.executeGsctl('build',
                                         self.constant.REBUILD_SUCCESS_MSG)
            self.log.info('---进行主备切换---')
            success_msg = self.constant.SWITCHOVER_SUCCESS_MSG
            result = self.pri_sh.executeGsctl('switchover', success_msg)
            self.log.info(result)
            result = self.pri_sh.exec_refresh_conf()
            self.log.info(result)

        self.log.info(f"-----{os.path.basename(__file__)} end-----")
